1 Summary
To examine the samplers’ uniformity, we ran them to generate samples corresponding to \(\alpha = 0.01, \beta = 0.01\), and \(w = 0.1\). The timeout for each sample generation was set to one hour. In total, 373.5 hours (15.56 days) of CPU time were needed for generating the samples (or reaching the timeout).
The samples are avialable at: 
2 Initialization
2.1 Importing packages and installing them if needed
REQUIRED_PACKAGES <-
c("tidyverse", "gridExtra", "grid", "directlabels", "ggrepel","splines",
"modelr", "scales", "kableExtra")
lapply(
REQUIRED_PACKAGES,
function(pkg) {
print(pkg)
if (system.file(package = pkg) == "") {
install.packages(pkg,
repos = "http://cran.us.r-project.org"
)
}
do.call("library", list(pkg))
}
)## -- Attaching packages --------------------------------------- tidyverse 1.3.1 --## v ggplot2 3.3.3 v purrr 0.3.4
## v tibble 3.1.2 v dplyr 1.0.6
## v tidyr 1.1.3 v stringr 1.4.0
## v readr 1.4.0 v forcats 0.5.1## -- Conflicts ------------------------------------------ tidyverse_conflicts() --
## x dplyr::filter() masks stats::filter()
## x dplyr::lag() masks stats::lag()##
## Attaching package: 'gridExtra'## The following object is masked from 'package:dplyr':
##
## combine##
## Attaching package: 'scales'## The following object is masked from 'package:purrr':
##
## discard## The following object is masked from 'package:readr':
##
## col_factor##
## Attaching package: 'kableExtra'## The following object is masked from 'package:dplyr':
##
## group_rowssessionInfo()## R version 4.1.0 (2021-05-18)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 19043)
##
## Matrix products: default
##
## locale:
## [1] LC_COLLATE=Spanish_Spain.1252 LC_CTYPE=Spanish_Spain.1252
## [3] LC_MONETARY=Spanish_Spain.1252 LC_NUMERIC=C
## [5] LC_TIME=Spanish_Spain.1252
##
## attached base packages:
## [1] splines grid stats graphics grDevices utils datasets
## [8] methods base
##
## other attached packages:
## [1] kableExtra_1.3.4 scales_1.1.1 modelr_0.1.8
## [4] ggrepel_0.9.1 directlabels_2021.1.13 gridExtra_2.3
## [7] forcats_0.5.1 stringr_1.4.0 dplyr_1.0.6
## [10] purrr_0.3.4 readr_1.4.0 tidyr_1.1.3
## [13] tibble_3.1.2 ggplot2_3.3.3 tidyverse_1.3.1
##
## loaded via a namespace (and not attached):
## [1] Rcpp_1.0.6 svglite_2.0.0 lubridate_1.7.10 assertthat_0.2.1
## [5] digest_0.6.27 utf8_1.2.1 R6_2.5.0 cellranger_1.1.0
## [9] backports_1.2.1 reprex_2.0.0 evaluate_0.14 httr_1.4.2
## [13] pillar_1.6.1 rlang_0.4.11 readxl_1.3.1 rstudioapi_0.13
## [17] jquerylib_0.1.4 rmarkdown_2.8 webshot_0.5.2 munsell_0.5.0
## [21] broom_0.7.6 compiler_4.1.0 xfun_0.23 pkgconfig_2.0.3
## [25] systemfonts_1.0.2 htmltools_0.5.1.1 tidyselect_1.1.1 quadprog_1.5-8
## [29] fansi_0.4.2 viridisLite_0.4.0 crayon_1.4.1 dbplyr_2.1.1
## [33] withr_2.4.2 jsonlite_1.7.2 gtable_0.3.0 lifecycle_1.0.0
## [37] DBI_1.1.1 magrittr_2.0.1 cli_3.0.0 stringi_1.6.1
## [41] fs_1.5.0 xml2_1.3.2 bslib_0.2.5.1 ellipsis_0.3.2
## [45] generics_0.1.0 vctrs_0.3.8 tools_4.1.0 glue_1.4.2
## [49] hms_1.1.0 yaml_2.2.1 colorspace_2.0-1 rvest_1.0.0
## [53] knitr_1.33 haven_2.4.1 sass_0.4.02.2 Model constants and variables initialization
MODELS_PATH <- "../data"
MODELS_EXTENSIONS <-
c("bdd", "kus", "quicksampler", "smarch", "spur", "unigen2")
models <- list.dirs(path = MODELS_PATH,
full.names = FALSE,
recursive = FALSE)
SAMPLER_COLORS <- c("#E91E63", "#7EBD5E", "#3388CC", "#FA9D0D", "#666666", "#FF6E66")
SPL_MODELS <- c("axtls", "busybox", "DellSPLOT", "embtoolkit-onlybool", "fiasco",
"jhipster", "LargeAutomotive", "toybox", "uClibc")
FORMATTED_SPL_MODELS <- c("axTLS", "BusyBox", "DellSPLOT", "EmbToolkit", "Fiasco",
"JHipster", "LargeAutomotive", "ToyBox", "uClibc")
SAMPLERS <- c("BDDSampler", "KUS", "QuickSampler", "Smarch", "Spur", "Unigen2")2.3 Auxiliary functions and variables for plots
pretty_name <- function(id) {
if (id == "bdd") {
"BDDSampler"
} else if (id == "quicksampler") {
"Quicksampler"
} else if (id == "smarch") {
"Smarch"
} else if (id == "spur") {
"Spur"
} else if (id == "unigen2") {
"Unigen2"
} else if (id == "kus") {
"KUS"
} else {
stop("error in pretty_name")
}
}
g_legend <- function(a.gplot){
tmp <- ggplot_gtable(ggplot_build(a.gplot))
leg <- which(sapply(tmp$grobs, function(x) x$name) == "guide-box")
legend <- tmp$grobs[[leg]]
legend
}
blank_plot <-
theme(axis.title.x=element_blank(),
axis.text.x=element_blank(),
axis.ticks.x=element_blank(),
axis.title.y=element_blank(),
axis.text.y=element_blank(),
axis.ticks.y=element_blank()
)2.4 Analysis result tibble initialization
analysis_results <-
tibble(model = character(),
bdd_jsd = numeric(),
bdd_p_value = numeric(),
kus_jsd = numeric(),
kus_p_value = numeric(),
quicksampler_jsd = numeric(),
quicksampler_p_value = numeric(),
smarch_jsd = numeric(),
smarch_p_value = numeric(),
spur_jsd = numeric(),
spur_p_value = numeric(),
unigen2_jsd = numeric(),
unigen2_p_value = numeric()
)3 The SFpC Goodness-Of-Fit Test
3.1 Kullback–Leibler divergence
For discrete probability distributions \(P\) and \(Q\) defined on the same probability space \(\mathcal{X}\), the Kullback–Leibler divergence [1] from \(Q\) to \(P\) is defined as:
\[D_\mathrm{KL}(P||Q)=\sum_{x \in \mathcal{X}} P(x)\mathrm{log}_2\Big{(}\frac{P(x)}{Q(x)}\Big{)}\]
For the extreme cases, where \(P(x)\) or \(Q(x)\) are equal to zero, the directions given by Drost(2018) are followed to compute a numerically stable Kullback–Leibler divergence.
kullback_leibler_divergence <- function(p, q) {
Q_adjustment <- 0.00001
kld <- function(x, y) {
if (x == 0) {
0
} else if (y == 0) {
x*log(x/Q_adjustment, base=2)
} else {
x*log(x/y, base=2)
}
}
sum(map2_dbl(p, q, kld))
}3.2 Jensen–Shannon divergence (also known as information radius and total divergence to the average)
The Jensen–Shannon divergence (JSD) is a symmetrized and smoothed version of the Kullback–Leibler divergence [1] defined as: \[\mathrm{JSD(P||Q)}=\frac{1}{2}D_\mathrm{KL}(P||M)+\frac{1}{2}D_\mathrm{KL}(Q||M)\] where \(M=\frac{1}{2}(P+Q)\)
jensen_shannon_divergence <- function(p, q) {
m <- (p+q)/2
jsd <- kullback_leibler_divergence(p,m)/2 + kullback_leibler_divergence(q,m)/2
ifelse(jsd<1, jsd, 1)
}Let \(s\) be the sample size, and \(m\) the number of elements in \(P\) that are neither zero nor one (e.g., the JHipster model has 45 features, there are seven core features and no dead features, so \(m=38\)). According to the proof given by Grosse et al. in Section 4.C of [2], \(2 s (\mathrm{ln} 2) D(F, P)\) has a \(\chi^2\) distribution with \(m-1\) degrees of freedom. As a result, a [3] built upon the statistic \(2 s (\mathrm{ln} 2) D(F, P)\) will help us to decide whether the sampler is uniform.
for (m in models) {
writeLines(str_c("Goodness of fit test of ", m))
# Create and initialize variables to store divergences and p-values
for (ext in MODELS_EXTENSIONS) {
eval(parse(text=str_c(ext, "_jsd <- NA")))
eval(parse(text=str_c(ext, "_p_value <- NA")))
}
# Import the model theoretical distribution
th_path <- str_c(MODELS_PATH,
"/",
m,
"/population_desc")
theoretical_distribution <- read_delim(
file = str_c(th_path, "/", m, ".satdist"),
delim = " ",
col_names = FALSE,
col_types = cols(
col_integer(),
col_character()
)
)
colnames(theoretical_distribution) <- c("feature_num", "absolute_freqs")
# Compute the theoretical probabilities
absolute_freqs <- gmp::as.bigz(theoretical_distribution$absolute_freqs)
total_solutions <- sum(absolute_freqs)
theoretical_distribution$probabilities <-
as.numeric(absolute_freqs/total_solutions)
total_solutions <- as.numeric(total_solutions)
theoretical_distribution <- theoretical_distribution %>%
select("feature_num", "probabilities")
# Analysis of each sampler
plot_index <- 0
histograms <- list()
for (ext in MODELS_EXTENSIONS) {
plot_index <- plot_index + 1
# Import sample
file_name <- str_c(MODELS_PATH,
"/",
m,
"/std_samples/",
m, "_satdist.", ext)
if (!file.exists(file_name)) {
empty_plot <-
ggplot()+
ggtitle(pretty_name(ext)) +
annotate(geom="text", x=3, y=30, label="<< time out >>",
color="#E91E63",
size=8) +
blank_plot
histograms[[plot_index]] <- empty_plot
next()
}
sample <- read_delim(
file = file_name,
delim = ";",
col_names = TRUE,
col_types = cols(
col_integer()
)
)
colnames(sample) <- "feature_num"
# Compute empirical frequencies
empirical_distribution <- sample %>%
count(feature_num) %>%
mutate(absolute_freqs = n) %>%
arrange(feature_num)
sample_size <- nrow(sample)
empirical_distribution$frequencies <-
empirical_distribution$absolute_freqs/sample_size
empirical_distribution <- empirical_distribution %>%
select(feature_num, frequencies)
# Join theoretical and empirical information
distributions <- left_join(
theoretical_distribution,
empirical_distribution,
by="feature_num")
# Set NA's in the empirical distribution to zero
distributions[is.na(distributions$frequencies),]$frequencies <- 0
# Remove rows with no configurations
distributions <- filter(distributions, probabilities>0)
# Get jensen shannon divergence
jsd <- jensen_shannon_divergence(distributions$probabilities,
distributions$frequencies)
# Get goodness-of-fit p-value
X2 <- 2*sample_size*log(2)*jsd
degrees_of_freedom <- nrow(distributions)-1
p_value <- 1-pchisq(X2, degrees_of_freedom)
p_value
# Store jsd and p_values in their corresponding variables
eval(parse(text=str_c(ext, "_jsd <- ", jsd)))
eval(parse(text=str_c(ext, "_p_value <- ", p_value)))
# Prepare distribution data for plotting
histogram <- gather(distributions,
probabilities, frequencies,
value="pr", key="Distribution")
levels <- c("probabilities", "frequencies")
histogram$Distribution <- factor(histogram$Distribution, levels=levels)
histogram$Distribution <- fct_recode(histogram$Distribution,
"Theoretical" = "probabilities",
"Empirical" = "frequencies"
)
# Generate the histogram
histograms[[plot_index]] <-
ggplot(histogram,
aes(x=feature_num, y=pr, fill=Distribution, col=Distribution)) +
scale_fill_manual(values=c("#3388CC", "#E91E63")) +
scale_color_manual(values=c("#3388CC", "#E91E63")) +
geom_col(alpha=0.5, size=0.1, position="dodge2") +
scale_x_continuous("#True variables per\nSAT-solution")+#,
scale_y_continuous("Probability")+
ggtitle(pretty_name(ext))
} # for (ext in MODELS_EXTENSIONS)
analysis_results <- analysis_results %>%
add_row(model = m,
bdd_jsd = bdd_jsd,
bdd_p_value = bdd_p_value,
kus_jsd = kus_jsd,
kus_p_value = kus_p_value,
quicksampler_jsd = quicksampler_jsd,
quicksampler_p_value = quicksampler_p_value,
smarch_jsd = smarch_jsd,
smarch_p_value = smarch_p_value,
spur_jsd = spur_jsd,
spur_p_value = spur_p_value,
unigen2_jsd = unigen2_jsd,
unigen2_p_value = unigen2_p_value
)
legend <- g_legend(histograms[[1]])
for (i in 1:6) {
histograms[[i]] <- histograms[[i]] +
theme(legend.position = "none")
}
histograms_plot <- arrangeGrob(
arrangeGrob(textGrob(
m,
gp = gpar(fontsize = 18, font=2)),
legend,
nrow=1, ncol=2,
widths=c(2,1)),
arrangeGrob(
histograms[[1]], histograms[[2]], histograms[[3]],
histograms[[4]], histograms[[5]], histograms[[6]],
nrow=2),
nrow=2,
heights=c(1,6)
)
grid.arrange(histograms_plot)
file_name <- str_c(MODELS_PATH,
"/",
m,
"/goodness_of_fit/",
m,
"_hist.pdf")
ggsave(file_name, histograms_plot, width=15, height=6)
grid.arrange(histograms_plot)
} # for (m in models)## Goodness of fit test of 10.sk_1_46
## Goodness of fit test of 107.sk_3_90
## Goodness of fit test of 109.sk_4_36
## Goodness of fit test of 110.sk_3_88
## Goodness of fit test of 111.sk_2_36
## Goodness of fit test of 19.sk_3_48
## Goodness of fit test of 27.sk_3_32
## Goodness of fit test of 32.sk_4_38
## Goodness of fit test of 35.sk_3_52
## Goodness of fit test of 53.sk_4_32
## Goodness of fit test of 55.sk_3_46
## Goodness of fit test of 77.sk_3_44
## Goodness of fit test of 84.sk_4_77
## Goodness of fit test of axtls
## Goodness of fit test of blasted_case_0_b11_1
## Goodness of fit test of blasted_case_0_b12_1
## Goodness of fit test of blasted_case_0_b12_2
## Goodness of fit test of blasted_case_1_b11_1
## Goodness of fit test of blasted_case_1_b12_1
## Goodness of fit test of blasted_case_1_b12_2
## Goodness of fit test of blasted_case_1_b14_1
## Goodness of fit test of blasted_case_1_b14_2
## Goodness of fit test of blasted_case_1_b14_3
## Goodness of fit test of blasted_case_2_b12_1
## Goodness of fit test of blasted_case_2_b12_2
## Goodness of fit test of blasted_case_2_b14_1
## Goodness of fit test of blasted_case_2_b14_2
## Goodness of fit test of blasted_case_2_b14_3
## Goodness of fit test of blasted_case_3_b14_1
## Goodness of fit test of blasted_case_3_b14_2
## Goodness of fit test of blasted_case_3_b14_3
## Goodness of fit test of blasted_case1
## Goodness of fit test of blasted_case10
## Goodness of fit test of blasted_case100
## Goodness of fit test of blasted_case101
## Goodness of fit test of blasted_case102
## Goodness of fit test of blasted_case103
## Goodness of fit test of blasted_case105
## Goodness of fit test of blasted_case106
## Goodness of fit test of blasted_case108
## Goodness of fit test of blasted_case109
## Goodness of fit test of blasted_case11
## Goodness of fit test of blasted_case110
## Goodness of fit test of blasted_case111
## Goodness of fit test of blasted_case112
## Goodness of fit test of blasted_case113
## Goodness of fit test of blasted_case114
## Goodness of fit test of blasted_case115
## Goodness of fit test of blasted_case116
## Goodness of fit test of blasted_case117
## Goodness of fit test of blasted_case118
## Goodness of fit test of blasted_case119
## Goodness of fit test of blasted_case120
## Goodness of fit test of blasted_case121
## Goodness of fit test of blasted_case122
## Goodness of fit test of blasted_case123
## Goodness of fit test of blasted_case124
## Goodness of fit test of blasted_case125
## Goodness of fit test of blasted_case126
## Goodness of fit test of blasted_case127
## Goodness of fit test of blasted_case128
## Goodness of fit test of blasted_case130
## Goodness of fit test of blasted_case131
## Goodness of fit test of blasted_case132
## Goodness of fit test of blasted_case133
## Goodness of fit test of blasted_case134
## Goodness of fit test of blasted_case135
## Goodness of fit test of blasted_case136
## Goodness of fit test of blasted_case137
## Goodness of fit test of blasted_case14
## Goodness of fit test of blasted_case144
## Goodness of fit test of blasted_case145
## Goodness of fit test of blasted_case146
## Goodness of fit test of blasted_case15
## Goodness of fit test of blasted_case17
## Goodness of fit test of blasted_case19
## Goodness of fit test of blasted_case2
## Goodness of fit test of blasted_case20
## Goodness of fit test of blasted_case200
## Goodness of fit test of blasted_case201
## Goodness of fit test of blasted_case202
## Goodness of fit test of blasted_case203
## Goodness of fit test of blasted_case204
## Goodness of fit test of blasted_case205
## Goodness of fit test of blasted_case206
## Goodness of fit test of blasted_case207
## Goodness of fit test of blasted_case208
## Goodness of fit test of blasted_case21
## Goodness of fit test of blasted_case210
## Goodness of fit test of blasted_case211
## Goodness of fit test of blasted_case213
## Goodness of fit test of blasted_case214
## Goodness of fit test of blasted_case22
## Goodness of fit test of blasted_case23
## Goodness of fit test of blasted_case24
## Goodness of fit test of blasted_case25
## Goodness of fit test of blasted_case26
## Goodness of fit test of blasted_case27
## Goodness of fit test of blasted_case28
## Goodness of fit test of blasted_case29
## Goodness of fit test of blasted_case3
## Goodness of fit test of blasted_case30
## Goodness of fit test of blasted_case31
## Goodness of fit test of blasted_case32
## Goodness of fit test of blasted_case33
## Goodness of fit test of blasted_case34
## Goodness of fit test of blasted_case35
## Goodness of fit test of blasted_case36
## Goodness of fit test of blasted_case38
## Goodness of fit test of blasted_case39
## Goodness of fit test of blasted_case4
## Goodness of fit test of blasted_case40
## Goodness of fit test of blasted_case41
## Goodness of fit test of blasted_case43
## Goodness of fit test of blasted_case44
## Goodness of fit test of blasted_case45
## Goodness of fit test of blasted_case46
## Goodness of fit test of blasted_case47
## Goodness of fit test of blasted_case5
## Goodness of fit test of blasted_case50
## Goodness of fit test of blasted_case51
## Goodness of fit test of blasted_case52
## Goodness of fit test of blasted_case53
## Goodness of fit test of blasted_case54
## Goodness of fit test of blasted_case55
## Goodness of fit test of blasted_case56
## Goodness of fit test of blasted_case57
## Goodness of fit test of blasted_case58
## Goodness of fit test of blasted_case59
## Goodness of fit test of blasted_case59_1
## Goodness of fit test of blasted_case6
## Goodness of fit test of blasted_case60
## Goodness of fit test of blasted_case61
## Goodness of fit test of blasted_case62
## Goodness of fit test of blasted_case63
## Goodness of fit test of blasted_case64
## Goodness of fit test of blasted_case68
## Goodness of fit test of blasted_case7
## Goodness of fit test of blasted_case8
## Goodness of fit test of blasted_case9
## Goodness of fit test of blasted_squaring22
## Goodness of fit test of blasted_squaring26
## Goodness of fit test of blasted_squaring50
## Goodness of fit test of blasted_squaring51
## Goodness of fit test of busybox
## Goodness of fit test of DellSPLOT
## Goodness of fit test of embtoolkit-onlybool
## Goodness of fit test of fiasco
## Goodness of fit test of GuidanceService2.sk_2_27
## Goodness of fit test of jhipster
## Goodness of fit test of LargeAutomotive
## Goodness of fit test of polynomial.sk_7_25
## Goodness of fit test of registerlesSwap.sk_3_10
## Goodness of fit test of s1196a_3_2
## Goodness of fit test of s1196a_7_4
## Goodness of fit test of s1238a_3_2
## Goodness of fit test of s1238a_7_4
## Goodness of fit test of s1488_15_7
## Goodness of fit test of s1488_7_4
## Goodness of fit test of s27_15_7
## Goodness of fit test of s27_3_2
## Goodness of fit test of s27_7_4
## Goodness of fit test of s27_new_15_7
## Goodness of fit test of s27_new_3_2
## Goodness of fit test of s27_new_7_4
## Goodness of fit test of s298_15_7
## Goodness of fit test of s298_3_2
## Goodness of fit test of s298_7_4
## Goodness of fit test of s344_15_7
## Goodness of fit test of s344_3_2
## Goodness of fit test of s344_7_4
## Goodness of fit test of s349_15_7
## Goodness of fit test of s349_3_2
## Goodness of fit test of s349_7_4
## Goodness of fit test of s382_15_7
## Goodness of fit test of s382_3_2
## Goodness of fit test of s382_7_4
## Goodness of fit test of s420_15_7
## Goodness of fit test of s420_3_2
## Goodness of fit test of s420_7_4
## Goodness of fit test of s420_new_15_7
## Goodness of fit test of s420_new_3_2
## Goodness of fit test of s420_new_7_4
## Goodness of fit test of s420_new1_15_7
## Goodness of fit test of s420_new1_3_2
## Goodness of fit test of s420_new1_7_4
## Goodness of fit test of s444_15_7
## Goodness of fit test of s444_3_2
## Goodness of fit test of s444_7_4
## Goodness of fit test of s510_15_7
## Goodness of fit test of s510_3_2
## Goodness of fit test of s510_7_4
## Goodness of fit test of s526_15_7
## Goodness of fit test of s526_3_2
## Goodness of fit test of s526_7_4
## Goodness of fit test of s526a_15_7
## Goodness of fit test of s526a_3_2
## Goodness of fit test of s526a_7_4
## Goodness of fit test of s641_15_7
## Goodness of fit test of s641_3_2
## Goodness of fit test of s641_7_4
## Goodness of fit test of s713_15_7
## Goodness of fit test of s713_3_2
## Goodness of fit test of s713_7_4
## Goodness of fit test of s820a_15_7
## Goodness of fit test of s820a_3_2
## Goodness of fit test of s820a_7_4
## Goodness of fit test of s832a_15_7
## Goodness of fit test of s832a_3_2
## Goodness of fit test of s832a_7_4
## Goodness of fit test of s838_15_7
## Goodness of fit test of s838_3_2
## Goodness of fit test of s838_7_4
## Goodness of fit test of s953a_3_2
## Goodness of fit test of s953a_7_4
## Goodness of fit test of tableBasedAddition.sk_240_1024
## Goodness of fit test of toybox
## Goodness of fit test of uClibc
4 General overview
overview <- analysis_results[,seq(1,13,by=2)]
colnames(overview)[2:7] <- SAMPLERS
rejected <- rep(0, 6)
for(i in 2:7) {
non_nan_p_values <- overview[[i]][!is.na(overview[[i]])]
rejected[i-1] <- 100*sum(non_nan_p_values<0.01)/length(non_nan_p_values)
}
rejected_summary <- tibble(
sampler = SAMPLERS,
rejected = str_c(round(rejected, 2), "% rejected")
)
for (i in 2:7) {
overview[[i]] <- cut(overview[[i]], breaks=seq(0, 1, by=0.1), include.lowest=TRUE, na.rm=TRUE)
}
overview <- overview %>%
gather(2:7, key="sampler", value="p_value") %>%
filter(!is.na(p_value))
models_per_sampler <- overview %>%
group_by(sampler) %>%
summarize(total=n())
p_values_per_sampler <- overview %>%
group_by(sampler, p_value) %>%
summarize(p_value_count=n())## `summarise()` has grouped output by 'sampler'. You can override using the `.groups` argument.summary_table <-
inner_join(models_per_sampler, p_values_per_sampler) %>%
mutate(percentage = 100*p_value_count/total) ## Joining, by = "sampler"levels <- c("[0,0.1]", "(0.1,0.2]", "(0.2,0.3]", "(0.3,0.4]", "(0.5,0.6]", "(0.6,0.7]",
"(0.7,0.8]", "(0.8,0.9]", "(0.9,1]")
summary_table$p_value <- factor(summary_table$p_value, levels=levels)
ggplot(summary_table, aes(x=p_value, y=percentage, fill=sampler)) +
scale_fill_manual(values=SAMPLER_COLORS) +
geom_bar(stat="identity", color="black", size=0.1) +
facet_wrap(.~sampler) +
geom_text(x = 5, y = 70, aes(label = rejected), data = rejected_summary, size=3) +
theme_bw() +
theme(legend.position = "none") +
theme(axis.text.x = element_text (angle=60, hjust=1)) +
scale_x_discrete("p-value") +
scale_y_continuous("%Samples") 
ggsave("test_results.pdf", width = 4.8, height = 3.5)5 Detailed SFpC test results
In the following table:
- bdd_jsd and bdd_p_value stand for the Jensen-Shanon divergences and p-values obtained with BDDSampler, respectively.
- kus_jsd and kus_p_value stand for the Jensen-Shanon divergences and p-values obtained with KUS, respectively.
- …
- unigen2_jsd and unigen2_p_value stand for the Jensen-Shanon divergences and p-values obtained with Unigen2, respectively.
write.table(
analysis_results,
file = str_c(MODELS_PATH,"/goodness_of_fit.csv"),
sep = ";",
row.names = FALSE
)
knitr::kable(analysis_results)| model | bdd_jsd | bdd_p_value | kus_jsd | kus_p_value | quicksampler_jsd | quicksampler_p_value | smarch_jsd | smarch_p_value | spur_jsd | spur_p_value | unigen2_jsd | unigen2_p_value |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 10.sk_1_46 | 0.0013316 | 1.0000000 | 0.0010786 | 1.0000000 | 0.1303615 | 0.0000000 | NA | NA | 0.0011061 | 1.0000000 | NA | NA |
| 107.sk_3_90 | 0.0011242 | 1.0000000 | 0.0011822 | 1.0000000 | 0.5392516 | 0.0000000 | NA | NA | 0.0011434 | 1.0000000 | NA | NA |
| 109.sk_4_36 | 0.0011793 | 1.0000000 | 0.0013787 | 1.0000000 | 0.4529571 | 0.0000000 | NA | NA | 0.0017584 | 1.0000000 | NA | NA |
| 110.sk_3_88 | 0.0016989 | 1.0000000 | NA | NA | 0.4763476 | 0.0000000 | NA | NA | 0.0020305 | 1.0000000 | NA | NA |
| 111.sk_2_36 | 0.0012871 | 0.9999995 | 0.0010419 | 1.0000000 | 0.6147990 | 0.0000000 | NA | NA | 0.0010214 | 1.0000000 | NA | NA |
| 19.sk_3_48 | 0.0014507 | 1.0000000 | 0.0014589 | 1.0000000 | 0.2438726 | 0.0000000 | NA | NA | 0.0014534 | 1.0000000 | NA | NA |
| 27.sk_3_32 | 0.0008328 | 0.9999999 | 0.0014774 | 0.9998948 | 0.2736768 | 0.0000000 | NA | NA | 0.0012461 | 0.9999872 | NA | NA |
| 32.sk_4_38 | 0.0014524 | 0.9999993 | 0.0011134 | 1.0000000 | 0.1098637 | 0.0000000 | NA | NA | 0.0010514 | 1.0000000 | NA | NA |
| 35.sk_3_52 | 0.0011267 | 1.0000000 | 0.0010216 | 1.0000000 | 0.0758962 | 0.0000000 | NA | NA | 0.0009918 | 1.0000000 | NA | NA |
| 53.sk_4_32 | 0.0016238 | 0.9999840 | 0.0007717 | 1.0000000 | 0.0703430 | 0.0000000 | NA | NA | 0.0016278 | 0.9999834 | NA | NA |
| 55.sk_3_46 | 0.0009962 | 1.0000000 | 0.0016969 | 1.0000000 | 0.2433804 | 0.0000000 | NA | NA | 0.0011450 | 1.0000000 | NA | NA |
| 77.sk_3_44 | 0.0015424 | 0.9999999 | 0.0010112 | 1.0000000 | 0.2816439 | 0.0000000 | NA | NA | 0.0013167 | 1.0000000 | NA | NA |
| 84.sk_4_77 | 0.0008614 | 1.0000000 | 0.0014515 | 1.0000000 | 0.4004079 | 0.0000000 | NA | NA | 0.0012513 | 1.0000000 | NA | NA |
| axtls | 0.0007275 | 1.0000000 | 0.0011498 | 1.0000000 | 0.3379597 | 0.0000000 | NA | NA | 0.0006899 | 1.0000000 | NA | NA |
| blasted_case_0_b11_1 | 0.0019278 | 1.0000000 | 0.0417133 | 0.0000000 | 0.0997540 | 0.0000000 | NA | NA | 0.0019610 | 1.0000000 | NA | NA |
| blasted_case_0_b12_1 | 0.0029611 | 1.0000000 | 0.4688336 | 0.0000000 | 0.2526064 | 0.0000000 | NA | NA | 0.0028220 | 1.0000000 | NA | NA |
| blasted_case_0_b12_2 | 0.0033852 | 1.0000000 | 0.0788632 | 0.0000000 | 0.0232519 | 0.0000013 | NA | NA | 0.0033285 | 1.0000000 | NA | NA |
| blasted_case_1_b11_1 | 0.0022914 | 1.0000000 | 0.1195219 | 0.0000000 | 0.0739621 | 0.0000000 | NA | NA | 0.0023048 | 1.0000000 | NA | NA |
| blasted_case_1_b12_1 | 0.0024868 | 1.0000000 | 0.3545621 | 0.0000000 | 0.0240123 | 0.0000001 | NA | NA | 0.0025873 | 1.0000000 | NA | NA |
| blasted_case_1_b12_2 | 0.0032931 | 1.0000000 | 0.2577323 | 0.0000000 | 0.0462589 | 0.0000000 | NA | NA | 0.0030283 | 1.0000000 | NA | NA |
| blasted_case_1_b14_1 | 0.0018851 | 1.0000000 | 0.0017869 | 1.0000000 | 0.4632492 | 0.0000000 | NA | NA | 0.0016259 | 1.0000000 | 0.0110824 | 1.0000000 |
| blasted_case_1_b14_2 | 0.0014742 | 1.0000000 | 0.0016105 | 1.0000000 | 0.4078432 | 0.0000000 | NA | NA | 0.0018392 | 1.0000000 | 0.0113253 | 1.0000000 |
| blasted_case_1_b14_3 | 0.0016818 | 1.0000000 | 0.0015429 | 1.0000000 | 0.5278118 | 0.0000000 | NA | NA | 0.0021431 | 1.0000000 | NA | NA |
| blasted_case_2_b12_1 | 0.0028778 | 1.0000000 | 0.3232143 | 0.0000000 | 0.0765531 | 0.0000000 | NA | NA | 0.0022554 | 1.0000000 | NA | NA |
| blasted_case_2_b12_2 | 0.0036842 | 1.0000000 | 0.1826947 | 0.0000000 | 0.0260535 | 0.0000008 | NA | NA | 0.0035877 | 1.0000000 | NA | NA |
| blasted_case_2_b14_1 | 0.0018851 | 1.0000000 | 0.0012670 | 1.0000000 | 0.3929843 | 0.0000000 | NA | NA | 0.0016464 | 1.0000000 | 0.0149023 | 1.0000000 |
| blasted_case_2_b14_2 | 0.0018100 | 1.0000000 | 0.0013717 | 1.0000000 | 0.3597543 | 0.0000000 | NA | NA | 0.0013334 | 1.0000000 | 0.0166888 | 1.0000000 |
| blasted_case_2_b14_3 | 0.0020891 | 1.0000000 | 0.0018365 | 1.0000000 | 0.1725337 | 0.0000000 | NA | NA | 0.0022225 | 1.0000000 | NA | NA |
| blasted_case_3_b14_1 | 0.0016591 | 1.0000000 | 0.0018600 | 1.0000000 | 0.4719370 | 0.0000000 | NA | NA | 0.0012666 | 1.0000000 | 0.0118623 | 1.0000000 |
| blasted_case_3_b14_2 | 0.0019442 | 1.0000000 | 0.0023708 | 1.0000000 | 0.3347211 | 0.0000000 | NA | NA | 0.0017235 | 1.0000000 | 0.0139701 | 1.0000000 |
| blasted_case_3_b14_3 | 0.0014045 | 1.0000000 | 0.0018063 | 1.0000000 | 0.5344324 | 0.0000000 | NA | NA | 0.0013614 | 1.0000000 | NA | NA |
| blasted_case1 | 0.0016826 | 1.0000000 | 0.0009507 | 1.0000000 | 0.1988378 | 0.8339750 | NA | NA | 0.0017658 | 0.9999999 | 0.0109720 | 1.0000000 |
| blasted_case10 | 0.0007979 | 1.0000000 | 0.0011938 | 1.0000000 | 0.3494668 | 0.0000000 | NA | NA | 0.0012424 | 1.0000000 | NA | NA |
| blasted_case100 | 0.0010896 | 1.0000000 | 0.0007843 | 1.0000000 | 0.0925407 | 0.2607446 | 0.0100337 | 1.0000000 | 0.0012345 | 1.0000000 | 0.0075227 | 1.0000000 |
| blasted_case101 | 0.0014500 | 1.0000000 | 0.0015283 | 1.0000000 | 0.1954709 | 0.0000000 | 0.0136956 | 1.0000000 | 0.0012776 | 1.0000000 | 0.0131078 | 1.0000000 |
| blasted_case102 | 0.0003854 | 0.9995032 | 0.0003133 | 0.9998448 | 0.0231344 | 0.9090618 | 0.0000286 | 1.0000000 | 0.0007476 | 0.9851516 | 0.0037826 | 0.9997615 |
| blasted_case103 | 0.0008751 | 0.9698405 | 0.0003510 | 0.9997050 | 0.0026038 | 0.9999994 | 0.0000286 | 1.0000000 | 0.0005778 | 0.9957505 | 0.0039667 | 0.9996884 |
| blasted_case105 | 0.0012287 | 1.0000000 | 0.0014819 | 1.0000000 | 0.6248556 | 0.0000000 | NA | NA | 0.0008722 | 1.0000000 | 0.0109131 | 1.0000000 |
| blasted_case106 | 0.0018708 | 1.0000000 | 0.0008649 | 1.0000000 | 0.5739558 | 0.0000000 | NA | NA | 0.0014795 | 1.0000000 | 0.0159217 | 1.0000000 |
| blasted_case108 | 0.0023707 | 1.0000000 | 0.0022013 | 1.0000000 | 0.0678261 | 0.0000000 | NA | NA | 0.0023988 | 1.0000000 | 0.0205054 | 1.0000000 |
| blasted_case109 | 0.0023712 | 1.0000000 | 0.0021783 | 1.0000000 | 0.1485599 | 0.0000000 | NA | NA | 0.0029915 | 1.0000000 | 0.0173750 | 1.0000000 |
| blasted_case11 | 0.0008591 | 0.9999999 | 0.0010693 | 0.9999983 | 0.0453019 | 0.9475705 | NA | NA | 0.0012212 | 0.9999901 | 0.0126112 | 0.9999667 |
| blasted_case110 | 0.0014911 | 0.9999998 | 0.0011653 | 1.0000000 | 0.0081234 | 0.0593937 | NA | NA | 0.0012856 | 1.0000000 | 0.0081647 | 1.0000000 |
| blasted_case111 | 0.0015777 | 0.9999996 | 0.0013906 | 1.0000000 | 0.0081556 | 0.0349344 | NA | NA | 0.0018419 | 0.9999949 | NA | NA |
| blasted_case112 | 0.0008276 | 1.0000000 | 0.2058916 | 0.0000000 | 0.0277496 | 0.0000000 | NA | NA | 0.0009899 | 0.9999998 | 0.0090163 | 0.9999994 |
| blasted_case113 | 0.0016337 | 0.9999997 | 0.0011378 | 1.0000000 | 0.0088883 | 0.0033368 | NA | NA | 0.0014250 | 1.0000000 | NA | NA |
| blasted_case114 | 0.0023924 | 1.0000000 | 0.0016931 | 1.0000000 | 0.3825535 | 0.0000000 | NA | NA | 0.0021159 | 1.0000000 | NA | NA |
| blasted_case115 | 0.0020164 | 1.0000000 | 0.0019272 | 1.0000000 | 0.7008073 | 0.0000000 | NA | NA | 0.0016360 | 1.0000000 | NA | NA |
| blasted_case116 | 0.0020569 | 1.0000000 | 0.4346648 | 0.0000000 | 0.1682556 | 0.0000000 | NA | NA | 0.0018352 | 1.0000000 | NA | NA |
| blasted_case117 | 0.0012065 | 1.0000000 | 0.0014203 | 1.0000000 | 0.0171544 | 0.0000000 | NA | NA | 0.0011701 | 1.0000000 | NA | NA |
| blasted_case118 | 0.0012810 | 1.0000000 | 0.0012636 | 1.0000000 | 0.0049857 | 0.6586274 | NA | NA | 0.0011136 | 1.0000000 | NA | NA |
| blasted_case119 | 0.0018581 | 1.0000000 | 0.0797089 | 0.0000000 | 0.4994645 | 0.0000000 | NA | NA | 0.0019467 | 1.0000000 | 0.0205294 | 1.0000000 |
| blasted_case120 | 0.0020628 | 1.0000000 | 0.0584869 | 0.0000000 | 0.2793107 | 0.0000000 | NA | NA | 0.0018295 | 1.0000000 | NA | NA |
| blasted_case121 | 0.0017096 | 1.0000000 | 0.0019320 | 1.0000000 | 0.0465179 | 0.0000000 | NA | NA | 0.0015757 | 1.0000000 | 0.0125992 | 1.0000000 |
| blasted_case122 | 0.0011936 | 1.0000000 | 0.0009375 | 1.0000000 | 0.2876590 | 0.0000000 | NA | NA | 0.0016608 | 0.9999977 | NA | NA |
| blasted_case123 | 0.0013649 | 1.0000000 | 0.0266899 | 0.0000000 | 0.3467390 | 0.0000000 | NA | NA | 0.0013202 | 1.0000000 | 0.0080562 | 1.0000000 |
| blasted_case124 | 0.0015303 | 1.0000000 | 0.0017748 | 1.0000000 | 0.4583823 | 0.0000000 | NA | NA | 0.0013057 | 1.0000000 | 0.0138454 | 1.0000000 |
| blasted_case125 | 0.0014193 | 1.0000000 | 0.0012320 | 1.0000000 | 0.3326160 | 0.0000000 | NA | NA | 0.0018344 | 1.0000000 | NA | NA |
| blasted_case126 | 0.0017141 | 1.0000000 | 0.0013343 | 1.0000000 | 0.2419164 | 0.0000000 | NA | NA | 0.0009272 | 1.0000000 | NA | NA |
| blasted_case127 | 0.0005416 | 0.9445866 | 0.0001098 | 0.9997753 | 0.0004725 | 1.0000000 | 0.0004725 | 1.0000000 | 0.0004186 | 0.9747380 | 0.0005179 | 0.9999831 |
| blasted_case128 | 0.0005416 | 0.9445866 | 0.0003419 | 0.9868785 | 0.0002259 | 1.0000000 | 0.0004725 | 1.0000000 | 0.0003239 | 0.9890396 | 0.0020782 | 0.9969058 |
| blasted_case130 | 0.0018911 | 1.0000000 | 0.2137364 | 0.0000000 | 0.2872276 | 0.0000000 | NA | NA | 0.0017778 | 1.0000000 | NA | NA |
| blasted_case131 | 0.0017268 | 1.0000000 | 0.0975177 | 0.0000000 | 0.4296307 | 0.0000000 | NA | NA | 0.0011836 | 1.0000000 | NA | NA |
| blasted_case132 | 0.0015379 | 1.0000000 | 0.0014369 | 1.0000000 | 0.3391403 | 0.0000000 | NA | NA | 0.0013685 | 1.0000000 | 0.0114168 | 1.0000000 |
| blasted_case133 | 0.0014676 | 1.0000000 | 0.0017505 | 1.0000000 | 0.4996645 | 0.0000000 | NA | NA | 0.0017825 | 1.0000000 | NA | NA |
| blasted_case134 | 0.0012031 | 1.0000000 | 0.0010800 | 1.0000000 | 0.2331239 | 0.0000000 | 0.0127711 | 1.0000000 | 0.0012302 | 1.0000000 | 0.0125423 | 1.0000000 |
| blasted_case135 | 0.0014394 | 1.0000000 | 0.0012609 | 1.0000000 | 0.3344309 | 0.0000000 | NA | NA | 0.0018227 | 1.0000000 | 0.0135920 | 1.0000000 |
| blasted_case136 | 0.0017052 | 1.0000000 | 0.0018395 | 1.0000000 | 0.4187843 | 0.0000000 | NA | NA | 0.0012336 | 1.0000000 | NA | NA |
| blasted_case137 | 0.0017052 | 0.9999986 | 0.0009536 | 1.0000000 | 0.0294039 | 0.0000048 | 0.0096525 | 1.0000000 | 0.0011012 | 1.0000000 | 0.0120360 | 1.0000000 |
| blasted_case14 | 0.0011852 | 1.0000000 | 0.0094691 | 0.0589483 | 0.0567743 | 0.0000000 | NA | NA | 0.0016727 | 1.0000000 | NA | NA |
| blasted_case144 | 0.0027271 | 1.0000000 | 0.0025736 | 1.0000000 | 0.4061649 | 0.0000000 | NA | NA | 0.0019940 | 1.0000000 | NA | NA |
| blasted_case145 | 0.0011384 | 1.0000000 | 0.0011936 | 1.0000000 | 0.2452657 | 0.0000000 | NA | NA | 0.0011247 | 1.0000000 | NA | NA |
| blasted_case146 | 0.0015591 | 1.0000000 | 0.0012580 | 1.0000000 | 0.1752916 | 0.0000000 | NA | NA | 0.0009134 | 1.0000000 | NA | NA |
| blasted_case15 | 0.0011690 | 1.0000000 | 0.0015610 | 1.0000000 | 0.1917139 | 0.0000000 | NA | NA | 0.0013028 | 1.0000000 | NA | NA |
| blasted_case17 | 0.0014954 | 0.9998125 | 0.0008796 | 0.9999998 | 0.0113537 | 0.9999996 | 0.0116591 | 0.9999962 | 0.0017960 | 0.9986162 | 0.0128243 | 0.9999850 |
| blasted_case19 | 0.0024373 | 1.0000000 | 0.0385241 | 0.0000000 | 0.1428813 | 0.0000000 | NA | NA | 0.0019256 | 1.0000000 | NA | NA |
| blasted_case2 | 0.0020627 | 1.0000000 | 0.0014367 | 1.0000000 | 0.1579749 | 0.0000000 | NA | NA | 0.0018493 | 1.0000000 | 0.0123748 | 1.0000000 |
| blasted_case20 | 0.0020343 | 1.0000000 | 0.0129630 | 0.2644008 | 0.1542216 | 0.0000000 | NA | NA | 0.0028414 | 1.0000000 | NA | NA |
| blasted_case200 | 0.0001413 | 0.9000586 | 0.0004033 | 0.6441214 | 0.0000000 | 1.0000000 | 0.1379254 | 0.9024450 | 0.0000902 | 0.9457864 | NA | NA |
| blasted_case201 | 0.0014536 | 1.0000000 | 0.0011061 | 1.0000000 | 0.1952627 | 0.0000000 | NA | NA | 0.0017182 | 1.0000000 | 0.0121129 | 1.0000000 |
| blasted_case202 | 0.0014536 | 1.0000000 | 0.0009882 | 1.0000000 | 0.3206113 | 0.0000000 | NA | NA | 0.0012026 | 1.0000000 | 0.0081080 | 1.0000000 |
| blasted_case203 | 0.0011079 | 1.0000000 | 0.0012247 | 1.0000000 | 0.3229443 | 0.0000000 | NA | NA | 0.0011089 | 1.0000000 | 0.0090918 | 1.0000000 |
| blasted_case204 | 0.0018557 | 1.0000000 | 0.0011121 | 1.0000000 | 0.3094139 | 0.0000000 | NA | NA | 0.0012490 | 1.0000000 | 0.0095621 | 1.0000000 |
| blasted_case205 | 0.0011079 | 1.0000000 | 0.0012486 | 1.0000000 | 0.3724481 | 0.0000000 | NA | NA | 0.0011552 | 1.0000000 | 0.0098149 | 1.0000000 |
| blasted_case206 | 0.0001421 | 0.8992123 | 0.0001195 | 0.9201797 | 0.0000000 | 1.0000000 | 0.1379254 | 0.9024450 | 0.0001959 | 0.8470039 | NA | NA |
| blasted_case207 | 0.0013431 | 1.0000000 | 0.0010460 | 1.0000000 | 0.5121875 | 0.0000000 | NA | NA | 0.0012930 | 1.0000000 | NA | NA |
| blasted_case208 | 0.0012305 | 1.0000000 | 0.0009939 | 1.0000000 | 0.5010501 | 0.0000000 | NA | NA | 0.0011204 | 1.0000000 | NA | NA |
| blasted_case21 | 0.0010137 | 0.9999972 | 0.0010786 | 0.9999939 | 0.0358161 | 0.9813413 | NA | NA | 0.0012047 | 0.9999763 | 0.0111999 | 0.9999781 |
| blasted_case210 | 0.0020923 | 1.0000000 | 0.0022516 | 1.0000000 | 0.5861683 | 0.0000000 | NA | NA | 0.0021938 | 1.0000000 | NA | NA |
| blasted_case211 | 0.0021476 | 1.0000000 | 0.0020937 | 1.0000000 | 0.6154459 | 0.0000000 | NA | NA | 0.0025250 | 1.0000000 | NA | NA |
| blasted_case213 | 0.0013722 | 1.0000000 | 0.0012998 | 1.0000000 | 0.2991038 | 0.0000000 | NA | NA | 0.0014452 | 1.0000000 | NA | NA |
| blasted_case214 | 0.0012945 | 1.0000000 | 0.0014664 | 1.0000000 | 0.5036780 | 0.0000000 | NA | NA | 0.0015989 | 1.0000000 | NA | NA |
| blasted_case22 | 0.0012935 | 0.9999447 | 0.0006666 | 1.0000000 | 0.1869638 | 0.0989761 | NA | NA | 0.0013351 | 0.9999200 | 0.0103280 | 0.9999919 |
| blasted_case23 | 0.0011576 | 0.9999915 | 0.0012776 | 0.9999709 | 0.0395712 | 0.9990099 | 0.0087868 | 0.9999999 | 0.0009228 | 0.9999996 | 0.0095131 | 0.9999997 |
| blasted_case24 | 0.0004206 | 0.9999989 | 0.0006107 | 0.9999688 | 0.0125787 | 0.9999985 | 0.0000716 | 1.0000000 | 0.0005936 | 0.9999756 | 0.0114299 | 0.9942758 |
| blasted_case25 | 0.0009693 | 0.9912006 | 0.0004414 | 0.9999515 | 0.1382340 | 0.2980760 | 0.0000069 | 1.0000000 | 0.0006286 | 0.9994241 | 0.0089516 | 0.9886393 |
| blasted_case26 | 0.0002250 | 0.9999973 | 0.0008145 | 0.9914855 | 0.0170156 | 0.8769486 | 0.0062134 | 0.9977760 | 0.0007287 | 0.9953640 | 0.0021523 | 0.9999972 |
| blasted_case27 | 0.0008119 | 0.9916353 | 0.0004896 | 0.9995547 | 0.0432248 | 0.5604750 | 0.0008315 | 1.0000000 | 0.0007625 | 0.9940458 | 0.0067173 | 0.9960794 |
| blasted_case28 | 0.0005210 | 0.9993479 | 0.0009358 | 0.9824850 | 0.0081984 | 0.9985542 | 0.0017588 | 0.9999994 | 0.0003331 | 0.9999622 | 0.0077963 | 0.9911335 |
| blasted_case29 | 0.0007240 | 0.9998686 | 0.0007213 | 0.9998726 | 0.0102261 | 0.9999999 | 0.0000716 | 1.0000000 | 0.0006946 | 0.9999070 | 0.0127154 | 0.9882940 |
| blasted_case3 | 0.0022203 | 1.0000000 | 0.0017513 | 1.0000000 | 0.3053524 | 0.0000000 | NA | NA | 0.0018026 | 1.0000000 | 0.0177045 | 1.0000000 |
| blasted_case30 | 0.0004174 | 0.9999677 | 0.0004714 | 0.9999221 | 0.0863573 | 0.3691331 | 0.0000069 | 1.0000000 | 0.0004367 | 0.9999551 | 0.0035992 | 0.9999729 |
| blasted_case31 | 0.0008331 | 0.9904030 | 0.0003728 | 0.9999210 | 0.0085349 | 0.9989658 | 0.0054084 | 0.9990193 | 0.0005700 | 0.9988783 | 0.0082956 | 0.9877055 |
| blasted_case32 | 0.0007719 | 0.9936328 | 0.0005398 | 0.9991911 | 0.0039624 | 0.9999843 | 0.0029465 | 0.9999796 | 0.0004580 | 0.9997063 | 0.0037487 | 0.9998859 |
| blasted_case33 | 0.0005342 | 0.9992410 | 0.0005246 | 0.9993203 | 0.1039727 | 0.0005682 | 0.0053979 | 0.9990307 | 0.0011252 | 0.9574154 | 0.0042879 | 0.9997317 |
| blasted_case34 | 0.0023581 | 1.0000000 | 0.0048159 | 0.9903900 | 0.4083584 | 0.0000000 | NA | NA | 0.0013987 | 1.0000000 | NA | NA |
| blasted_case35 | 0.0019160 | 1.0000000 | 0.0011863 | 1.0000000 | 0.1477567 | 0.0000000 | NA | NA | 0.0020315 | 1.0000000 | NA | NA |
| blasted_case36 | 0.0006208 | 0.9939175 | 0.2326812 | 0.0000000 | 0.0051919 | 0.9999619 | 0.0018140 | 0.9999996 | 0.0005916 | 0.9952163 | 0.0082272 | 0.9540883 |
| blasted_case38 | 0.0008133 | 0.9999977 | 0.0011353 | 0.9999046 | 0.0689595 | 0.9915912 | 0.0000006 | 1.0000000 | 0.0006154 | 0.9999999 | 0.0105105 | 0.9994004 |
| blasted_case39 | 0.0007194 | 1.0000000 | 0.1606357 | 0.0000000 | 0.2939446 | 0.0000000 | NA | NA | 0.0011126 | 1.0000000 | NA | NA |
| blasted_case4 | 0.0012104 | 0.9999997 | 0.0012092 | 0.9999997 | 0.0947795 | 0.5528688 | NA | NA | 0.0022558 | 0.9983168 | 0.0113389 | 0.9999999 |
| blasted_case40 | 0.0010076 | 1.0000000 | 0.1644924 | 0.0000000 | 0.2403773 | 0.0000000 | NA | NA | 0.0011101 | 1.0000000 | NA | NA |
| blasted_case41 | 0.0007194 | 1.0000000 | 0.1660845 | 0.0000000 | 0.2335599 | 0.0000000 | NA | NA | 0.0012302 | 1.0000000 | NA | NA |
| blasted_case43 | 0.0011632 | 0.9999947 | 0.0007628 | 1.0000000 | 0.1568203 | 0.0080741 | NA | NA | 0.0011212 | 0.9999968 | 0.0096724 | 0.9999979 |
| blasted_case44 | 0.0010830 | 0.9999998 | 0.0010262 | 0.9999999 | 0.2041506 | 0.2177300 | NA | NA | 0.0013682 | 0.9999942 | 0.0082610 | 0.9999998 |
| blasted_case45 | 0.0007650 | 1.0000000 | 0.0012163 | 0.9999906 | 0.1112878 | 0.5943415 | NA | NA | 0.0011523 | 0.9999954 | 0.0075755 | 0.9999999 |
| blasted_case46 | 0.0013910 | 0.9999956 | 0.0013299 | 0.9999977 | 0.2104658 | 0.8968999 | NA | NA | 0.0007067 | 1.0000000 | 0.0088611 | 0.9999998 |
| blasted_case47 | 0.0012693 | 0.9998826 | 0.0012506 | 0.9999002 | 0.7102742 | 0.0000000 | NA | NA | 0.0004985 | 1.0000000 | 0.0065104 | 0.9999997 |
| blasted_case5 | 0.0008064 | 1.0000000 | 0.0013215 | 1.0000000 | 0.0456971 | 0.0000000 | NA | NA | 0.0009391 | 1.0000000 | 0.0063748 | 1.0000000 |
| blasted_case50 | 0.0024623 | 1.0000000 | 0.0021160 | 1.0000000 | 0.3571804 | 0.0000000 | NA | NA | 0.0024799 | 1.0000000 | NA | NA |
| blasted_case51 | 0.0008130 | 0.9999998 | 0.0010555 | 0.9999953 | 0.4176279 | 0.0000000 | NA | NA | 0.0011509 | 0.9999864 | 0.0077257 | 0.9999989 |
| blasted_case52 | 0.0013778 | 0.9998852 | 0.0011439 | 0.9999874 | 0.2779868 | 0.0000000 | NA | NA | 0.0011367 | 0.9999883 | 0.0065585 | 0.9999999 |
| blasted_case53 | 0.0013778 | 0.9998852 | 0.0007551 | 0.9999999 | 0.0865416 | 0.7954681 | NA | NA | 0.0007449 | 1.0000000 | 0.0074541 | 0.9999993 |
| blasted_case54 | 0.0016609 | 1.0000000 | 0.0017487 | 1.0000000 | 0.2636373 | 0.0000000 | NA | NA | 0.0016475 | 1.0000000 | 0.0139244 | 1.0000000 |
| blasted_case55 | 0.0014842 | 0.9999998 | 0.0016966 | 0.9999979 | 0.4148133 | 0.9282980 | NA | NA | 0.0011513 | 1.0000000 | 0.0083362 | 1.0000000 |
| blasted_case56 | 0.0017046 | 1.0000000 | 0.0016037 | 1.0000000 | 0.0408963 | 0.0000000 | NA | NA | 0.0012138 | 1.0000000 | 0.0134205 | 1.0000000 |
| blasted_case57 | 0.0013478 | 1.0000000 | 0.0510739 | 0.0000000 | 0.1266625 | 0.0000000 | NA | NA | 0.0008728 | 1.0000000 | 0.0097858 | 1.0000000 |
| blasted_case58 | 0.0010272 | 0.9999432 | 0.0006516 | 0.9999996 | 0.0181738 | 0.9787982 | 0.0067619 | 0.9999985 | 0.0006744 | 0.9999995 | 0.0040251 | 1.0000000 |
| blasted_case59 | 0.0010423 | 0.9998872 | 0.1516599 | 0.0000000 | 0.1345081 | 0.0082167 | 0.0112438 | 0.9994616 | 0.0009698 | 0.9999455 | 0.0102664 | 0.9997502 |
| blasted_case59_1 | 0.0013802 | 0.9984114 | 0.1459362 | 0.0000000 | 0.0216792 | 0.9673182 | 0.0078154 | 0.9999858 | 0.0008659 | 0.9999831 | 0.0054736 | 0.9999997 |
| blasted_case6 | 0.0019353 | 1.0000000 | 0.0016612 | 1.0000000 | 0.3291754 | 0.0000000 | NA | NA | 0.0021152 | 1.0000000 | NA | NA |
| blasted_case60 | 0.0002232 | 0.9595804 | 0.0002519 | 0.9477296 | 0.0000000 | 1.0000000 | 0.0319773 | 0.9848368 | 0.0004032 | 0.8665128 | NA | NA |
| blasted_case61 | 0.0014460 | 1.0000000 | 0.0134048 | 0.0000572 | 0.2103115 | 0.0000000 | NA | NA | 0.0011016 | 1.0000000 | NA | NA |
| blasted_case62 | 0.0016002 | 1.0000000 | 0.0013240 | 1.0000000 | 0.1184163 | 0.0000000 | NA | NA | 0.0020601 | 0.9999981 | 0.0082011 | 1.0000000 |
| blasted_case63 | 0.0008590 | 0.9999917 | 0.2602539 | 0.0000000 | 0.0279455 | 0.9351566 | 0.0061343 | 0.9999995 | 0.0008255 | 0.9999946 | 0.0065191 | 0.9999988 |
| blasted_case64 | 0.0005776 | 0.9999996 | 0.0006396 | 0.9999987 | 0.0136229 | 0.0000000 | NA | NA | 0.0008974 | 0.9999556 | 0.0107610 | 0.9992996 |
| blasted_case68 | 0.0007683 | 1.0000000 | 0.0240921 | 0.0000000 | 0.1492359 | 0.1379597 | NA | NA | 0.0013657 | 1.0000000 | 0.0124779 | 1.0000000 |
| blasted_case7 | 0.0007730 | 1.0000000 | 0.0011280 | 0.9999989 | 0.1434573 | 0.0000044 | NA | NA | 0.0010251 | 0.9999997 | 0.0047736 | 1.0000000 |
| blasted_case8 | 0.0012732 | 1.0000000 | 0.0013943 | 1.0000000 | 0.0493863 | 0.9999862 | NA | NA | 0.0012684 | 1.0000000 | 0.0110409 | 1.0000000 |
| blasted_case9 | 0.0012663 | 1.0000000 | 0.0012100 | 1.0000000 | 0.0766977 | 0.0000000 | NA | NA | 0.0011743 | 1.0000000 | NA | NA |
| blasted_squaring22 | 0.0016196 | 1.0000000 | NA | NA | 0.1697998 | 0.0000000 | NA | NA | 0.0013724 | 1.0000000 | NA | NA |
| blasted_squaring26 | 0.0037136 | 1.0000000 | 0.8371974 | 0.0000000 | 0.5721998 | 0.0000000 | NA | NA | 0.0033812 | 1.0000000 | NA | NA |
| blasted_squaring50 | 0.0027198 | 1.0000000 | 0.0690323 | 0.0000000 | 0.1661534 | 0.0000000 | NA | NA | 0.0026105 | 1.0000000 | NA | NA |
| blasted_squaring51 | 0.0026033 | 1.0000000 | 0.1503676 | 0.0000000 | 0.3244400 | 0.0000000 | NA | NA | 0.0025400 | 1.0000000 | NA | NA |
| busybox | 0.0010854 | 1.0000000 | 0.0016437 | 1.0000000 | 1.0000000 | 0.0000000 | NA | NA | 0.0011415 | 1.0000000 | NA | NA |
| DellSPLOT | 0.0003014 | 0.9961108 | 0.0004283 | 0.9855897 | 0.2338054 | 0.8461812 | NA | NA | 0.0002471 | 0.9982106 | 0.0032351 | 0.9619242 |
| embtoolkit-onlybool | 0.0013525 | 1.0000000 | NA | NA | 0.9884235 | 1.0000000 | NA | NA | 0.5273388 | 0.0000000 | NA | NA |
| fiasco | 0.0008427 | 0.9999933 | 0.0006547 | 0.9999996 | 0.9867368 | 0.2969574 | NA | NA | 0.0009342 | 0.9999792 | NA | NA |
| GuidanceService2.sk_2_27 | 0.0016772 | 1.0000000 | 0.0013097 | 1.0000000 | 0.5229459 | 0.0000000 | NA | NA | 0.0017600 | 1.0000000 | NA | NA |
| jhipster | 0.0005612 | 0.9997344 | 0.0009312 | 0.9930437 | 0.0430483 | 0.0000000 | 0.0055432 | 0.9998684 | 0.0007651 | 0.9979174 | 0.0078519 | 0.9985606 |
| LargeAutomotive | 0.0010105 | 1.0000000 | NA | NA | 1.0000000 | 0.0000000 | NA | NA | 0.0012346 | 1.0000000 | NA | NA |
| polynomial.sk_7_25 | 0.0007879 | 0.9999938 | 0.0013146 | 0.9989725 | 0.1717230 | 0.9999104 | NA | NA | 0.0005736 | 0.9999998 | NA | NA |
| registerlesSwap.sk_3_10 | 0.0013104 | 0.9999947 | 0.4208369 | 0.0000000 | 0.0812949 | 0.9999998 | NA | NA | 0.0010537 | 0.9999998 | NA | NA |
| s1196a_3_2 | 0.0019010 | 1.0000000 | 0.0015319 | 1.0000000 | 0.0174924 | 0.0000000 | NA | NA | 0.0018479 | 1.0000000 | NA | NA |
| s1196a_7_4 | 0.0018194 | 1.0000000 | 0.0017421 | 1.0000000 | 0.0170746 | 0.0044829 | NA | NA | 0.0018878 | 1.0000000 | NA | NA |
| s1238a_3_2 | 0.0016355 | 1.0000000 | 0.3349503 | 0.0000000 | 0.0792596 | 0.0000000 | NA | NA | 0.0016896 | 1.0000000 | NA | NA |
| s1238a_7_4 | 0.0019527 | 1.0000000 | 0.0970689 | 0.0000000 | 0.0302714 | 0.0000000 | NA | NA | 0.0022505 | 1.0000000 | NA | NA |
| s1488_15_7 | 0.0039171 | 1.0000000 | 0.0024831 | 1.0000000 | 0.0046338 | 1.0000000 | NA | NA | 0.0029006 | 1.0000000 | NA | NA |
| s1488_7_4 | 0.0032145 | 1.0000000 | 0.0022372 | 1.0000000 | 0.0074809 | 0.9997675 | NA | NA | 0.0020517 | 1.0000000 | NA | NA |
| s27_15_7 | 0.0001402 | 0.9949001 | 0.0003224 | 0.9550624 | 0.0000000 | 1.0000000 | 0.0010081 | 0.9999820 | 0.0002767 | 0.9692023 | 0.0071316 | 0.7893139 |
| s27_3_2 | 0.0000499 | 0.9997297 | 0.0004916 | 0.8809164 | 0.0000000 | 1.0000000 | 0.0010081 | 0.9999820 | 0.0005180 | 0.8667991 | 0.0028538 | 0.9737092 |
| s27_7_4 | 0.0006485 | 0.7900245 | 0.0002599 | 0.9737350 | 0.0000000 | 1.0000000 | 0.0010081 | 0.9999820 | 0.0001219 | 0.9965380 | 0.0070115 | 0.7961063 |
| s27_new_15_7 | 0.0005223 | 0.8644580 | 0.0002265 | 0.9815599 | 0.0000000 | 1.0000000 | 0.0025033 | 0.9999150 | 0.0003687 | 0.9379580 | 0.0060043 | 0.8715722 |
| s27_new_3_2 | 0.0006655 | 0.7793464 | 0.0007565 | 0.7208783 | 0.0000000 | 1.0000000 | 0.0025033 | 0.9999150 | 0.0010005 | 0.5622203 | 0.0063797 | 0.8540243 |
| s27_new_7_4 | 0.0005223 | 0.8644580 | 0.0003133 | 0.9580963 | 0.0000000 | 1.0000000 | 0.0025033 | 0.9999150 | 0.0001216 | 0.9965628 | 0.0102811 | 0.6451722 |
| s298_15_7 | 0.0032591 | 1.0000000 | 0.0026927 | 1.0000000 | 0.0417725 | 0.0000000 | NA | NA | 0.0021713 | 1.0000000 | 0.0229020 | 1.0000000 |
| s298_3_2 | 0.0012855 | 0.9999686 | 0.0005079 | 1.0000000 | 0.0387272 | 0.0000000 | NA | NA | 0.0013307 | 0.9999523 | 0.0092254 | 0.9999474 |
| s298_7_4 | 0.0014466 | 0.9999999 | 0.0009939 | 1.0000000 | 0.0069437 | 0.0850060 | NA | NA | 0.0015550 | 0.9999995 | 0.0071903 | 1.0000000 |
| s344_15_7 | 0.0029505 | 1.0000000 | 0.0021591 | 1.0000000 | 0.0492569 | 0.0000000 | NA | NA | 0.0015445 | 1.0000000 | 0.0175614 | 1.0000000 |
| s344_3_2 | 0.0010539 | 1.0000000 | 0.0014445 | 1.0000000 | 0.0477145 | 0.0000000 | NA | NA | 0.0012916 | 1.0000000 | 0.0108842 | 1.0000000 |
| s344_7_4 | 0.0011222 | 1.0000000 | 0.0016653 | 1.0000000 | 0.0620987 | 0.0000000 | NA | NA | 0.0011098 | 1.0000000 | 0.0144409 | 1.0000000 |
| s349_15_7 | 0.0023320 | 1.0000000 | 0.0018605 | 1.0000000 | 0.0396747 | 0.0000000 | NA | NA | 0.0019910 | 1.0000000 | 0.0168187 | 1.0000000 |
| s349_3_2 | 0.0014079 | 1.0000000 | 0.0008473 | 1.0000000 | 0.0317825 | 0.0000000 | NA | NA | 0.0013047 | 1.0000000 | 0.0090775 | 1.0000000 |
| s349_7_4 | 0.0013145 | 1.0000000 | 0.0016555 | 1.0000000 | 0.0878584 | 0.0000000 | NA | NA | 0.0013237 | 1.0000000 | 0.0109518 | 1.0000000 |
| s382_15_7 | 0.0022877 | 1.0000000 | 0.0029245 | 1.0000000 | 0.0925399 | 0.0000000 | NA | NA | 0.0019491 | 1.0000000 | NA | NA |
| s382_3_2 | 0.0011023 | 0.9999974 | 0.0008544 | 0.9999999 | 0.0342812 | 0.0000000 | NA | NA | 0.0009291 | 0.9999997 | 0.0096036 | 0.9998449 |
| s382_7_4 | 0.0009355 | 1.0000000 | 0.0011428 | 1.0000000 | 0.1626970 | 0.0000000 | NA | NA | 0.0010347 | 1.0000000 | 0.0054702 | 1.0000000 |
| s420_15_7 | 0.0020879 | 1.0000000 | 0.0017349 | 1.0000000 | 0.8076502 | 0.0000000 | NA | NA | 0.0018350 | 1.0000000 | NA | NA |
| s420_3_2 | 0.0012572 | 1.0000000 | 0.0012299 | 1.0000000 | 0.1325137 | 0.0000000 | NA | NA | 0.0007952 | 1.0000000 | 0.0080565 | 1.0000000 |
| s420_7_4 | 0.0014360 | 1.0000000 | 0.0012755 | 1.0000000 | 0.5136831 | 0.0000000 | NA | NA | 0.0012335 | 1.0000000 | NA | NA |
| s420_new_15_7 | 0.0018205 | 1.0000000 | 0.0017795 | 1.0000000 | 0.8642182 | 0.0000000 | NA | NA | 0.0016005 | 1.0000000 | NA | NA |
| s420_new_3_2 | 0.0008616 | 1.0000000 | 0.0010368 | 1.0000000 | 0.3542817 | 0.0000000 | NA | NA | 0.0010032 | 1.0000000 | 0.0081586 | 1.0000000 |
| s420_new_7_4 | 0.0013269 | 1.0000000 | 0.0016775 | 1.0000000 | 0.4351491 | 0.0000000 | NA | NA | 0.0011869 | 1.0000000 | NA | NA |
| s420_new1_15_7 | 0.0021372 | 1.0000000 | 0.0018534 | 1.0000000 | 0.8187227 | 0.0000000 | NA | NA | 0.0018961 | 1.0000000 | NA | NA |
| s420_new1_3_2 | 0.0007740 | 1.0000000 | 0.0010303 | 1.0000000 | 0.4596475 | 0.0000000 | NA | NA | 0.0013100 | 1.0000000 | 0.0083575 | 1.0000000 |
| s420_new1_7_4 | 0.0014197 | 1.0000000 | 0.0013661 | 1.0000000 | 0.4152869 | 0.0000000 | NA | NA | 0.0012239 | 1.0000000 | NA | NA |
| s444_15_7 | 0.0025486 | 1.0000000 | 0.0018473 | 1.0000000 | 0.0391142 | 0.0000000 | NA | NA | 0.0022436 | 1.0000000 | NA | NA |
| s444_3_2 | 0.0010966 | 0.9999999 | 0.0010123 | 1.0000000 | 0.0291043 | 0.0000000 | NA | NA | 0.0010265 | 1.0000000 | 0.0058442 | 1.0000000 |
| s444_7_4 | 0.0014838 | 1.0000000 | 0.0015919 | 1.0000000 | 0.0577748 | 0.0000000 | NA | NA | 0.0017753 | 0.9999999 | NA | NA |
| s510_15_7 | 0.0014740 | 1.0000000 | 0.0012146 | 1.0000000 | 0.2817584 | 0.0000000 | NA | NA | 0.0008571 | 1.0000000 | NA | NA |
| s510_3_2 | 0.0015700 | 0.9999976 | 0.0014125 | 0.9999996 | 0.5610223 | 0.0000000 | NA | NA | 0.0010583 | 1.0000000 | 0.0055833 | 1.0000000 |
| s510_7_4 | 0.0011362 | 1.0000000 | 0.0014843 | 0.9999994 | 0.6334515 | 0.0000000 | NA | NA | 0.0007364 | 1.0000000 | NA | NA |
| s526_15_7 | 0.0019668 | 1.0000000 | 0.1423881 | 0.0000000 | 0.3579287 | 0.0000000 | NA | NA | 0.0021603 | 1.0000000 | NA | NA |
| s526_3_2 | 0.0011863 | 1.0000000 | 0.0014572 | 1.0000000 | 0.0700828 | 0.0000000 | NA | NA | 0.0013871 | 1.0000000 | NA | NA |
| s526_7_4 | 0.0016381 | 1.0000000 | 0.0015038 | 1.0000000 | 0.0947843 | 0.0000000 | NA | NA | 0.0018834 | 1.0000000 | NA | NA |
| s526a_15_7 | 0.0024347 | 1.0000000 | 0.4287505 | 0.0000000 | 0.3972689 | 0.0000000 | NA | NA | 0.0029217 | 1.0000000 | NA | NA |
| s526a_3_2 | 0.0013885 | 1.0000000 | 0.0020875 | 0.9999993 | 0.0780826 | 0.0000000 | NA | NA | 0.0009820 | 1.0000000 | NA | NA |
| s526a_7_4 | 0.0020097 | 1.0000000 | 0.0013201 | 1.0000000 | 0.1999664 | 0.0000000 | NA | NA | 0.0018324 | 1.0000000 | NA | NA |
| s641_15_7 | 0.0016424 | 1.0000000 | 0.0016678 | 1.0000000 | 0.0761401 | 0.0000000 | NA | NA | 0.0018326 | 1.0000000 | NA | NA |
| s641_3_2 | 0.0015115 | 1.0000000 | 0.0014608 | 1.0000000 | 0.2513842 | 0.0000000 | NA | NA | 0.0015287 | 1.0000000 | NA | NA |
| s641_7_4 | 0.0015445 | 1.0000000 | 0.0012454 | 1.0000000 | 0.0695304 | 0.0000000 | NA | NA | 0.0014589 | 1.0000000 | NA | NA |
| s713_15_7 | 0.0018409 | 1.0000000 | 0.0022917 | 1.0000000 | 0.3510273 | 0.0000000 | NA | NA | 0.0021685 | 1.0000000 | NA | NA |
| s713_3_2 | 0.0013083 | 1.0000000 | 0.0013266 | 1.0000000 | 0.0461828 | 0.0000000 | NA | NA | 0.0014265 | 1.0000000 | NA | NA |
| s713_7_4 | 0.0014334 | 1.0000000 | 0.0012237 | 1.0000000 | 0.0515399 | 0.0000000 | NA | NA | 0.0016370 | 1.0000000 | NA | NA |
| s820a_15_7 | 0.0022197 | 1.0000000 | 0.0020287 | 1.0000000 | 0.3824322 | 0.0000000 | NA | NA | 0.0027158 | 1.0000000 | NA | NA |
| s820a_3_2 | 0.0016802 | 0.9999739 | 0.0006194 | 1.0000000 | 0.2090161 | 0.0000000 | NA | NA | 0.0010361 | 1.0000000 | NA | NA |
| s820a_7_4 | 0.0012469 | 1.0000000 | 0.0012230 | 1.0000000 | 0.2343982 | 0.0000000 | NA | NA | 0.0014683 | 0.9999999 | NA | NA |
| s832a_15_7 | 0.0019201 | 1.0000000 | 0.0028947 | 1.0000000 | 0.4346639 | 0.0000000 | NA | NA | 0.0021888 | 1.0000000 | NA | NA |
| s832a_3_2 | 0.0014557 | 1.0000000 | 0.0014439 | 1.0000000 | 0.0551578 | 0.0000000 | NA | NA | 0.0010952 | 1.0000000 | NA | NA |
| s832a_7_4 | 0.0016064 | 1.0000000 | 0.0009699 | 1.0000000 | 0.4412826 | 0.0000000 | NA | NA | 0.0018659 | 1.0000000 | NA | NA |
| s838_15_7 | 0.0024001 | 1.0000000 | 0.0021370 | 1.0000000 | 0.7384079 | 0.0000000 | NA | NA | 0.0024456 | 1.0000000 | NA | NA |
| s838_3_2 | 0.0012048 | 1.0000000 | 0.0011324 | 1.0000000 | 0.3422012 | 0.0000000 | NA | NA | 0.0010543 | 1.0000000 | NA | NA |
| s838_7_4 | 0.0011270 | 1.0000000 | 0.0009914 | 1.0000000 | 0.3631068 | 0.0000000 | NA | NA | 0.0011334 | 1.0000000 | NA | NA |
| s953a_3_2 | 0.0011704 | 1.0000000 | 0.0013556 | 1.0000000 | 0.1260736 | 0.0000000 | NA | NA | 0.0007519 | 1.0000000 | NA | NA |
| s953a_7_4 | 0.0014209 | 1.0000000 | 0.0011910 | 1.0000000 | 0.3234640 | 0.0000000 | NA | NA | 0.0015761 | 1.0000000 | NA | NA |
| tableBasedAddition.sk_240_1024 | 0.0006929 | 1.0000000 | 0.0007742 | 1.0000000 | 0.2113487 | 0.0000000 | NA | NA | 0.0008411 | 1.0000000 | NA | NA |
| toybox | 0.0013390 | 1.0000000 | 0.0009941 | 1.0000000 | 0.2907412 | 0.0000000 | NA | NA | 0.0008472 | 1.0000000 | NA | NA |
| uClibc | 0.0011507 | 1.0000000 | 0.0008992 | 1.0000000 | 1.0000000 | 1.0000000 | NA | NA | 0.0010862 | 1.0000000 | NA | NA |
References
[1]
T. M. Cover and J. A. Thomas, Elements of information theory. Wiley, 2006.
[2]
I. Grosse, P. Bernaola-Galvan, P. Carpena, R. Roman-Roldan, J. Oliver, and H. E. Stanley, “Analysis of symbolic sequences using the Jensen-Shannon divergence,” Physical Review E, vol. 65, no. 2, pp. 041905/1–0419051/16, 2002.
[3]
R. B. D’Agostino and M. A. Stephens, Goodness-of-fit-techniques. CRC Press, 1986.